{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86158\\AppData\\Roaming\\Python\\Python38\\site-packages\\pandas\\core\\computation\\expressions.py:20: UserWarning: Pandas requires version '2.7.3' or newer of 'numexpr' (version '2.7.1' currently installed).\n",
      "  from pandas.core.computation.check import NUMEXPR_INSTALLED\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第1页找到 20 条评论\n",
      "第1页成功提取 20 条评论\n",
      "第2页找到 20 条评论\n",
      "第2页成功提取 20 条评论\n",
      "第3页找到 20 条评论\n",
      "第3页成功提取 20 条评论\n",
      "第4页找到 20 条评论\n",
      "第4页成功提取 20 条评论\n",
      "第5页找到 20 条评论\n",
      "第5页成功提取 20 条评论\n",
      "\n",
      "总共提取到 100 条评论\n",
      "        名字  评级                 评论时间 评论地点  \\\n",
      "0     Aom゜  力荐  2024-02-23 15:28:22   四川   \n",
      "1      大戴戴  力荐  2024-02-24 00:54:45   山东   \n",
      "2    調色盤壞了  力荐  2024-02-23 21:04:39   上海   \n",
      "3       小卷  力荐  2024-02-23 15:27:11   湖北   \n",
      "4      局外人  力荐  2024-02-23 11:55:29   浙江   \n",
      "..     ...  ..                  ...  ...   \n",
      "95     唐源源  力荐  2024-11-17 09:14:11   福建   \n",
      "96  adam43  还行  2024-06-08 14:25:37   北京   \n",
      "97      酸奶  力荐  2024-07-23 20:21:27   江苏   \n",
      "98       荼  还行  2024-07-28 21:48:37   北京   \n",
      "99      老猫  还行  2024-07-23 07:53:46   北京   \n",
      "\n",
      "                                                 评论内容  页码  \n",
      "0                                    唯一一部广告都不舍得快进的综艺。   1  \n",
      "1                              不就五个星星吗，又不是要天上的月亮，给他们！   1  \n",
      "2   今年救援零次，光这一句话，含金量太高了。。沉甸甸的成长。。（还是人均犟种，都发高烧了，嘴里必...   1  \n",
      "3   开局依旧是地狱模式，但是少年们已经不是从前的他们了，变得更好更强了。\\n镜头和剪辑比第一季更...   1  \n",
      "4                        第二季，做大做强的路上！因为还会期待你们的第三第四季呀！   1  \n",
      "..                                                ...  ..  \n",
      "95  这一季的第一集让我看得很舒适，剪辑和观众之间也有了默契。用狗狗给田命名，离谱中带着一丝合理。...   5  \n",
      "96  十个小伙子还是不错的。但节目初心早就忘了吧，各种资源，各种赞助。咱就是说，如果刨去粉丝向的收...   5  \n",
      "97  20250515二刷完，麦田音乐节给我半夜看兴奋了，发了一堆微博。  和第一季的音乐节比起来...   5  \n",
      "98  这季上的项目太多了，不仅感觉大家精力上顾不过来，镜头背后很多工作其实不是完全靠自己完成的，而...   5  \n",
      "99                                         实在看不动广告了……   5  \n",
      "\n",
      "[100 rows x 6 columns]\n",
      "\n",
      "数据统计：\n",
      "总评论数: 100\n",
      "有评级的评论数: 97\n",
      "包含地点的评论数: 100\n"
     ]
    }
   ],
   "source": [
    "# 初始化数据列表\n",
    "all_comments_data = []\n",
    "\n",
    "for i in range(5):\n",
    "    try:\n",
    "        url = 'https://movie.douban.com/subject/36571420/comments?start={}&limit=20&status=P&sort=new_score'.format(20*i)\n",
    "        headers = {\n",
    "            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36 Edg/137.0.0.0',\n",
    "            'Referer': 'https://movie.douban.com/subject/36571420/',\n",
    "            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'\n",
    "        }\n",
    "        \n",
    "        # 添加随机延时，避免请求过于频繁\n",
    "        time.sleep(random.uniform(1, 3))\n",
    "        \n",
    "        response = requests.get(url, headers=headers)\n",
    "        \n",
    "        # 检查请求是否成功\n",
    "        if response.status_code != 200:\n",
    "            print(f\"第{i+1}页请求失败，状态码: {response.status_code}\")\n",
    "            continue\n",
    "            \n",
    "        # 解析网页\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        \n",
    "        # 检查是否找到了评论容器\n",
    "        mod_bd = soup.find('div', class_='mod-bd')\n",
    "        if mod_bd is None:\n",
    "            print(f\"第{i+1}页：未找到评论容器，可能是反爬机制或页面结构变化\")\n",
    "            print(\"页面标题:\", soup.find('title'))\n",
    "            continue\n",
    "            \n",
    "        comment_items = mod_bd.find_all('div', class_='comment-item')\n",
    "        \n",
    "        if not comment_items:\n",
    "            print(f\"第{i+1}页：未找到评论项\")\n",
    "            continue\n",
    "            \n",
    "        print(f\"第{i+1}页找到 {len(comment_items)} 条评论\")\n",
    "        \n",
    "        # 提取当前页的评论信息\n",
    "        page_comments_data = []\n",
    "        \n",
    "        for comment_item in comment_items:\n",
    "            try:\n",
    "                # 提取评论者名字\n",
    "                comment_info = comment_item.find('span', class_='comment-info')\n",
    "                if comment_info:\n",
    "                    name_link = comment_info.find('a')\n",
    "                    name = name_link.text.strip() if name_link else '未知用户'\n",
    "                else:\n",
    "                    # 备用方法：直接查找所有包含people的链接\n",
    "                    people_links = comment_item.find_all('a', href=lambda x: x and '/people/' in x)\n",
    "                    name = people_links[0].text.strip() if people_links else '未知用户'\n",
    "                \n",
    "                # 评级\n",
    "                rating_span = comment_item.find('span', class_='rating')\n",
    "                if rating_span and 'title' in rating_span.attrs:\n",
    "                    rating = rating_span['title']\n",
    "                else:\n",
    "                    rating = '无评级'\n",
    "                \n",
    "                # 评论时间\n",
    "                time_span = comment_item.find('span', class_='comment-time')\n",
    "                if time_span:\n",
    "                    comment_time = time_span.get('title', '').strip() or time_span.text.strip()\n",
    "                else:\n",
    "                    comment_time = ''\n",
    "                \n",
    "                # 评论地点\n",
    "                location_span = comment_item.find('span', class_='comment-location')\n",
    "                comment_location = location_span.text.strip() if location_span else ''\n",
    "                \n",
    "                # 评论内容\n",
    "                content_span = comment_item.find('span', class_='short')\n",
    "                comment_content = content_span.text.strip() if content_span else ''\n",
    "                \n",
    "                page_comments_data.append({\n",
    "                    '名字': name,\n",
    "                    '评级': rating,\n",
    "                    '评论时间': comment_time,\n",
    "                    '评论地点': comment_location,\n",
    "                    '评论内容': comment_content,\n",
    "                    '页码': i+1\n",
    "                })\n",
    "                \n",
    "            except Exception as e:\n",
    "                print(f\"处理单条评论时出错: {e}\")\n",
    "                continue\n",
    "        \n",
    "        # 将当前页的数据添加到总数据中\n",
    "        all_comments_data.extend(page_comments_data)\n",
    "        print(f\"第{i+1}页成功提取 {len(page_comments_data)} 条评论\")\n",
    "        \n",
    "    except Exception as e:\n",
    "        print(f\"处理第{i+1}页时发生错误: {e}\")\n",
    "        continue\n",
    "\n",
    "# 创建DataFrame\n",
    "if all_comments_data:\n",
    "    df = pd.DataFrame(all_comments_data)\n",
    "    \n",
    "    # 显示结果\n",
    "    print(f\"\\n总共提取到 {len(df)} 条评论\")\n",
    "    print(df)\n",
    "    \n",
    "    # 统计信息\n",
    "    print(f\"\\n数据统计：\")\n",
    "    print(f\"总评论数: {len(df)}\")\n",
    "    print(f\"有评级的评论数: {len(df[df['评级'] != '无评级'])}\")\n",
    "    print(f\"包含地点的评论数: {len(df[df['评论地点'] != ''])}\")\n",
    "    \n",
    "else:\n",
    "    print(\"没有提取到任何评论数据\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
